from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

print "The purpose of this program is to prepare the file for computing the KL probability"

'''
print "Checking the real probability first"
# Checking mechanism
# Purpose: check whether the true probability is summing up to 1
# option1(The following file is not completed):
# the probability index is 1
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheirTrueProbablityAndPredictedProbablity"

# option2:
# the probability index is 2
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/realFreqOfTermsIn_100KQueries_2_4%_sortedByQueryTerm_withProbablityAdded"
inputFileHandler = open(inputFileName,"r")
totalTrueProbability = 0.0
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    trueProbability = float( lineElements[2] )
    totalTrueProbability += trueProbability
print "totalTrueProbability:",totalTrueProbability
inputFileHandler.close()
exit(1)
'''


'''
# Purpose of this part of logic: add the our 2D probability model into the final output evaluation file
NUM_QUERY_TERM_POSITIONS = 351734
# This dict is used for filter out some query terms which has freq >= 20
queryTermWithRealFreqIn85KQueriesDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%_sortedByQueryTermFreq"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    queryTermFreq = int(lineElements[1])
    queryTermWithRealFreqIn85KQueriesDict[queryTerm] = queryTermFreq

print "This dict contains the query terms with their real freq in 85K training queries"
print "len(queryTermWithRealFreqIn85KQueriesDict):",len(queryTermWithRealFreqIn85KQueriesDict)
inputFileHandler.close()

# key: eg. ROW1_0, ROW1_1
# value: (lowerBoundSmallRangeID, upperBoundSmallRangeID) eg. [0,4]
classLabelList = ["ROW1","ROW2","ROW3","ROW4","ROW5","SUM"]

# key: ROW1_6
# value: [0,73]
queryTermsAndRealProbablityDistributionDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/realFreqOfTermsIn_100KQueries_2_4%_sortedByQueryTerm_withProbablityAdded"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    probabilityInTestingQueries = float( lineElements[2] )
    
    # This statement will include all the query terms
    if queryTerm not in queryTermsAndRealProbablityDistributionDict:
        queryTermsAndRealProbablityDistributionDict[queryTerm] = probabilityInTestingQueries
    else:
        pass

print "example key: 'infants' value:",queryTermsAndRealProbablityDistributionDict["infants"]
print "example key: 'of' value:",queryTermsAndRealProbablityDistributionDict["of"]
print "len(queryTermsAndRealProbablityDistributionDict):",len(queryTermsAndRealProbablityDistributionDict)

print "temp check in..."
totalTempProbability = 0.0
for queryTerm in queryTermsAndRealProbablityDistributionDict:
    totalTempProbability += queryTermsAndRealProbablityDistributionDict[queryTerm]
print "totalTempProbability:",totalTempProbability
print "temp check out."

print 
inputFileHandler.close()

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/probabilityTableUsingProfIdea20130420_fixed"
inputFileHandler = open(inputFileName,"r")
dataLine = inputFileHandler.readline()
while not dataLine.strip().startswith("table:probability"):
    dataLine = inputFileHandler.readline()
# print "mark1:"
# print dataLine
# print "mark2:"
inputFileHandler.readline()
inputFileHandler.readline()

cellCorrespondingRangesDict = {}

for classLabel in classLabelList:
    rowLineData = inputFileHandler.readline().strip()
    rowLineDataElements = rowLineData.split(" ")
    for i in range(0,20):
        cellKey = classLabel + "_" + str(i)
        if cellKey not in cellCorrespondingRangesDict:
            cellCorrespondingRangesDict[cellKey] = rowLineDataElements[i+1].split(":")[0]

print "example key:ROW1_6 value:[0,73]"
print "len(cellCorrespondingRangesDict):",len(cellCorrespondingRangesDict)
print 

# print "cellCorrespondingRangesDict:",cellCorrespondingRangesDict
inputFileHandler.close()


# Load the final probability we want, NOT the intermediate probability from the 3rd table
cellCorrespondingProbabilityDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/probabilityWeWantFinally20130418"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    # Current way
    cellKey = lineElements[0]
    cellKeyProbabilityValue = float( lineElements[1] )
    if cellKey not in cellCorrespondingProbabilityDict:
        cellCorrespondingProbabilityDict[cellKey] = cellKeyProbabilityValue
    else:
        print "Unexpected Behaviour"
        exit(1)
    
    # OLD way of generating cellKey
    #for classLabel in classLabelList:
    #    for i in range(0,20):
    #        cellKey = classLabel + "_" + str(i)
    

print "example key:ROW1_6 value:",cellCorrespondingProbabilityDict["ROW1_6"]
print "len(cellCorrespondingProbabilityDict):",len(cellCorrespondingProbabilityDict)
print

inputFileHandler.close()


# number of small cells: 20000
smallCellWithAssociateQueryTermList = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/100KQueries_1_10%_with_query_terms"
inputFileHandler = open(inputFileName)
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    smallCellKey = lineElements[0]
    numOfQueryTerms = int(lineElements[1])
    smallCellWithAssociateQueryTermList[smallCellKey] = []
    for i in range(0,numOfQueryTerms):
        smallCellWithAssociateQueryTermList[smallCellKey].append( lineElements[i+2] )

# print smallCellWithAssociateQueryTermList
print "example key:'19_118' value:['diarrhea'] "
print "len(smallCellWithAssociateQueryTermList):",len(smallCellWithAssociateQueryTermList)
print 
inputFileHandler.close()


# key: the query term itself
# value: probability
queryTermWithAssociatedProbabilityDict = {}

# Let's work on the following data structures.
# cellCorrespondingRangesDict = {}
# cellCorrespondingProbabilityDict = {}
classLabelList2 = ["ROW1","ROW2","ROW3","ROW4","ROW5"]
for classLabel in classLabelList2:
    for i in range(0,20):
        cellKey = classLabel + "_" + str(i)
        lowerBoundRangeID = int( cellCorrespondingRangesDict[cellKey].split("[")[1].split("]")[0].split(",")[0] )
        upperBoundRangeID = int( cellCorrespondingRangesDict[cellKey].split("[")[1].split("]")[0].split(",")[1] )
        probability = float( cellCorrespondingProbabilityDict[cellKey] )
        # for debug ONLY
        # print cellKey,lowerBoundRangeID,upperBoundRangeID
        currentCellQueryTermList = []
        for j in range(lowerBoundRangeID,upperBoundRangeID+1):
            smallCellKey = str(i) + "_" + str(j)
            currentCellQueryTermList += smallCellWithAssociateQueryTermList[smallCellKey]
        # for debug ONLY
        # print cellKey,len(currentCellQueryTermList),currentCellQueryTermList
        # do the probablity assignment
        for queryTerm in currentCellQueryTermList:
            if queryTerm not in queryTermWithAssociatedProbabilityDict:
                queryTermWithAssociatedProbabilityDict[queryTerm] = probability
            else:
                print "mark3, NOT GOOD"
                exit(1)

print "example key:'richardson' value:0.436241610738,"
print "Those query terms are appeared in the 10K query justification set and with freq < 20"
print "len(queryTermWithAssociatedProbabilityDict):",len(queryTermWithAssociatedProbabilityDict)
print 
# print "queryTermWithAssociatedProbabilityDict['genetics']:",queryTermWithAssociatedProbabilityDict['genetics']

# do the set intersection thing
queryTermsFrom4KQueriesSet = set(queryTermsAndRealProbablityDistributionDict)
queryTermsFrom10KQueriesSet = set(queryTermWithAssociatedProbabilityDict)
newTermSet = queryTermsFrom4KQueriesSet.difference(queryTermsFrom10KQueriesSet)
intersectionSet = queryTermsFrom4KQueriesSet.intersection(queryTermsFrom10KQueriesSet)

print "len(queryTermsFrom4KQueriesSet):",len(queryTermsFrom4KQueriesSet)
# This newTermSet should be further divided into two parts
print "# of unseen query terms which has freq < 20(part2) and freq >= 20(part3):",len(newTermSet)
print "# of seen query terms which has freq less than or equal to 19(part1, can be handled):",len(intersectionSet)

# pre-load the query term with its freq in the collection
queryTermWithFreqInCollectionDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freqInCollection = int(lineElements[1])
    if queryTerm not in queryTermWithFreqInCollectionDict:
        queryTermWithFreqInCollectionDict[queryTerm] = freqInCollection
    else:
        print "Unexpected Behaviour"
        exit(1)
print "len(queryTermWithFreqInCollectionDict):",len(queryTermWithFreqInCollectionDict)
inputFileHandler.close()

# step1: deal with the unseen query terms, need to give them one of the 5 ranges
# direct comparison with the length of the inverted list

# smallRangeID beginningLength,endingLength 
# [0,4]:       [1,100)            0.0000115262492427
# [5,9]:       [100,665)          0.000660982699686
# [10,25]:     [665,2473)         0.00474471376998
# [26,64]:     [2473,9964)        0.013327935727
# [65,999]:    [9964,25205179]    0.0274554562171
# [0,999]:     [1,25205179]       0.0000571680709718

queryTermAndProfProbablityDistributionDict = {}

counterForOutOfLexiconTerms = 0
counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 = 0
counterForTermsWhichHaveSeenFromThe10KQueries = 0
counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20 = 0
# For the new/unseen query terms
for queryTerm in newTermSet:
    if queryTerm not in queryTermWithRealFreqIn85KQueriesDict or queryTermWithRealFreqIn85KQueriesDict[queryTerm] < 20:
        
        if queryTermWithFreqInCollectionDict[queryTerm] >= 1 and queryTermWithFreqInCollectionDict[queryTerm] < 100:
            # final probability with the cellKey = "ROW1_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 2.7962236405996843e-10
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 100 and queryTermWithFreqInCollectionDict[queryTerm] < 665:
            # final probability with the cellKey = "ROW2_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 1.6040397629079132e-08
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 665 and queryTermWithFreqInCollectionDict[queryTerm] < 2473:
            # final probability with the cellKey = "ROW3_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 1.1538218688794478e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 2473 and queryTermWithFreqInCollectionDict[queryTerm] < 9964:
            # final probability with the cellKey = "ROW4_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 3.2578867212223025e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 9964 and queryTermWithFreqInCollectionDict[queryTerm] <= 25205179:
            # final probability with the cellKey = "ROW5_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 8.031888903939055e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        else:
            # solution1
            # print "OutOfLexiconQueryTerm:",queryTerm
            
            # solution2
            # the rational behind this is: out of lexicon query terms has the smallest probability I can assign them.
            # the cellKey == "ROW1_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 2.7962236405996843e-10
            counterForOutOfLexiconTerms += 1
    else:
        # print "Popular query terms:",queryTerm,queryTermWithRealFreqIn85KQueriesDict[queryTerm]
        queryTermAndProfProbablityDistributionDict[queryTerm] = queryTermWithRealFreqIn85KQueriesDict[queryTerm] / NUM_QUERY_TERM_POSITIONS
        counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20 += 1

# For the query terms which have been seen from the training file
for queryTerm in intersectionSet:
    # directly copy the probability from queryTermWithAssociatedProbabilityDict to queryTermAndProfProbablityDistributionDict
    queryTermAndProfProbablityDistributionDict[queryTerm] = queryTermWithAssociatedProbabilityDict[queryTerm]
    counterForTermsWhichHaveSeenFromThe10KQueries += 1
       
print "counterForOutOfLexiconTerms:",counterForOutOfLexiconTerms
print "counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20:",counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20
print "counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20:",counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20
print "counterForTermsWhichHaveSeenFromThe10KQueries:",counterForTermsWhichHaveSeenFromThe10KQueries

print "len(queryTermAndProfProbablityDistributionDict):",len(queryTermAndProfProbablityDistributionDict)
print "len(queryTermsAndRealProbablityDistributionDict):",len(queryTermsAndRealProbablityDistributionDict)

# The output component     
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheirTrueProbablityAndPredictedProbablity"
outputFileHandler = open(outputFileName,"w")
for queryTerm in queryTermsAndRealProbablityDistributionDict:
    # + " " + str( queryTermsAndRealProbablityDistributionDict[queryTerm] )
    # + " " + str( queryTermAndProfProbablityDistributionDict[queryTerm] )
    outputFileHandler.write(queryTerm + " " + str( queryTermsAndRealProbablityDistributionDict[queryTerm] ) + " " + str( queryTermAndProfProbablityDistributionDict[queryTerm] ) + "\n")
outputFileHandler.close()
'''



# Purpose: add the values of a specific probability model into the final output evaluation file to see our performance
NUM_QUERY_TERM_POSITIONS = 351734
# This dict is used for filter out some query terms which has freq >= 20
queryTermWithRealFreqInTrainingQueriesDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    queryTermFreq = int(lineElements[1])
    if queryTerm not in queryTermWithRealFreqInTrainingQueriesDict:
        queryTermWithRealFreqInTrainingQueriesDict[queryTerm] = queryTermFreq
    else:
        print "Unexpected Bahaviour,Mark2"
        exit(1)

print "This dict contains the query terms with their real freq in 85K training queries"
print "len(queryTermWithRealFreqInTrainingQueriesDict):",len(queryTermWithRealFreqInTrainingQueriesDict)
inputFileHandler.close()

# key: eg. ROW1_0, ROW1_1
# value: (lowerBoundSmallRangeID, upperBoundSmallRangeID) eg. [0,4]
classLabelList = ["ROW1","ROW2","ROW3","ROW4","ROW5","SUM"]

# key: ROW1_6
# value: [0,73]
queryTermsAndRealProbablityDistributionDict = {}
# option1
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/realFreqOfTermsIn_100KQueries_2_4%_sortedByQueryTerm_withProbablityAdded"
# option2
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsIn_100KQueries_withProbablityAdded"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    probabilityInTestingQueries = float( lineElements[2] )
    
    # This statement will include all the query terms
    if queryTerm not in queryTermsAndRealProbablityDistributionDict:
        queryTermsAndRealProbablityDistributionDict[queryTerm] = probabilityInTestingQueries
    else:
        pass

print "example key: 'infants' value:",queryTermsAndRealProbablityDistributionDict["infants"]
print "example key: 'of' value:",queryTermsAndRealProbablityDistributionDict["of"]
print "len(queryTermsAndRealProbablityDistributionDict):",len(queryTermsAndRealProbablityDistributionDict)

print "temp check in..."
totalTempProbability = 0.0
for queryTerm in queryTermsAndRealProbablityDistributionDict:
    totalTempProbability += queryTermsAndRealProbablityDistributionDict[queryTerm]
print "totalTempProbability:",totalTempProbability
print "temp check out."

print 
inputFileHandler.close()

# Purpose of the following part logic is to assign the probabilities from our probability model(no matter 1D or 2D) to the test query terms
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/probabilityTableUsingProfIdea20130420_fixed"
inputFileHandler = open(inputFileName,"r")
dataLine = inputFileHandler.readline()
while not dataLine.strip().startswith("table:probability"):
    dataLine = inputFileHandler.readline()
# print "mark1:"
# print dataLine
# print "mark2:"
inputFileHandler.readline()
inputFileHandler.readline()

cellCorrespondingRangesDict = {}

for classLabel in classLabelList:
    rowLineData = inputFileHandler.readline().strip()
    rowLineDataElements = rowLineData.split(" ")
    for i in range(0,20):
        cellKey = classLabel + "_" + str(i)
        if cellKey not in cellCorrespondingRangesDict:
            cellCorrespondingRangesDict[cellKey] = rowLineDataElements[i+1].split(":")[0]

print "example key:ROW1_6 value:[0,73]"
print "len(cellCorrespondingRangesDict):",len(cellCorrespondingRangesDict)
print 

# print "cellCorrespondingRangesDict:",cellCorrespondingRangesDict
inputFileHandler.close()


# Load the final probability we want, NOT the intermediate probability from the 3rd table
cellCorrespondingProbabilityDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/probabilityWeWantFinally20130418"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    # Current way
    cellKey = lineElements[0]
    cellKeyProbabilityValue = float( lineElements[1] )
    if cellKey not in cellCorrespondingProbabilityDict:
        cellCorrespondingProbabilityDict[cellKey] = cellKeyProbabilityValue
    else:
        print "Unexpected Behaviour"
        exit(1)
    
    # OLD way of generating cellKey
    #for classLabel in classLabelList:
    #    for i in range(0,20):
    #        cellKey = classLabel + "_" + str(i)
    

print "example key:ROW1_6 value:",cellCorrespondingProbabilityDict["ROW1_6"]
print "len(cellCorrespondingProbabilityDict):",len(cellCorrespondingProbabilityDict)
print

inputFileHandler.close()


# The above logic is to do the 2D probability filling, the following is to change them ALL into 1D probability filling
# comment out the following sentences: 2D
# NOT comment out the following sentence: 1D
'''
classLabelNeededToBeReplacedList = ["ROW1","ROW2","ROW3","ROW4","ROW5"]
for label in classLabelNeededToBeReplacedList:
    for i in range(0,20):
        cellKey = label + "_" + str(i)
        cellKeyOnRightSide = "SUM" + "_" + str(i)
        cellCorrespondingProbabilityDict[cellKey] = cellCorrespondingProbabilityDict[cellKeyOnRightSide]
'''

# number of small cells: 20000
smallCellWithAssociateQueryTermList = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/100KQueries_1_10%_with_query_terms"
inputFileHandler = open(inputFileName)
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    smallCellKey = lineElements[0]
    numOfQueryTerms = int(lineElements[1])
    smallCellWithAssociateQueryTermList[smallCellKey] = []
    for i in range(0,numOfQueryTerms):
        smallCellWithAssociateQueryTermList[smallCellKey].append( lineElements[i+2] )

# print smallCellWithAssociateQueryTermList
print "example key:'19_118' value:['diarrhea'] "
print "len(smallCellWithAssociateQueryTermList):",len(smallCellWithAssociateQueryTermList)
print 
inputFileHandler.close()


# key: the query term itself
# value: probability
queryTermWithAssociatedProbabilityDict = {}

# Let's work on the following data structures.
# cellCorrespondingRangesDict = {}
# cellCorrespondingProbabilityDict = {}
classLabelList2 = ["ROW1","ROW2","ROW3","ROW4","ROW5"]
for classLabel in classLabelList2:
    for i in range(0,20):
        cellKey = classLabel + "_" + str(i)
        lowerBoundRangeID = int( cellCorrespondingRangesDict[cellKey].split("[")[1].split("]")[0].split(",")[0] )
        upperBoundRangeID = int( cellCorrespondingRangesDict[cellKey].split("[")[1].split("]")[0].split(",")[1] )
        probability = float( cellCorrespondingProbabilityDict[cellKey] )
        # for debug ONLY
        # print cellKey,lowerBoundRangeID,upperBoundRangeID
        currentCellQueryTermList = []
        for j in range(lowerBoundRangeID,upperBoundRangeID+1):
            smallCellKey = str(i) + "_" + str(j)
            currentCellQueryTermList += smallCellWithAssociateQueryTermList[smallCellKey]
        # for debug ONLY
        # print cellKey,len(currentCellQueryTermList),currentCellQueryTermList
        # do the probablity assignment
        for queryTerm in currentCellQueryTermList:
            if queryTerm not in queryTermWithAssociatedProbabilityDict:
                queryTermWithAssociatedProbabilityDict[queryTerm] = probability
            else:
                print "mark3, NOT GOOD"
                exit(1)

print "example key:'richardson' value:0.436241610738,"
print "Those query terms are appeared in the 10K query justification set and with freq < 20"
print "len(queryTermWithAssociatedProbabilityDict):",len(queryTermWithAssociatedProbabilityDict)
print 
# print "queryTermWithAssociatedProbabilityDict['genetics']:",queryTermWithAssociatedProbabilityDict['genetics']

# do the set intersection thing
queryTermsFrom4KQueriesSet = set(queryTermsAndRealProbablityDistributionDict)
queryTermsFrom10KQueriesSet = set(queryTermWithAssociatedProbabilityDict)
newTermSet = queryTermsFrom4KQueriesSet.difference(queryTermsFrom10KQueriesSet)
intersectionSet = queryTermsFrom4KQueriesSet.intersection(queryTermsFrom10KQueriesSet)

print "len(queryTermsFrom4KQueriesSet):",len(queryTermsFrom4KQueriesSet)
# This newTermSet should be further divided into two parts
print "# of unseen query terms which has freq < 20(part2) and freq >= 20(part3):",len(newTermSet)
print "# of seen query terms which has freq less than or equal to 19(part1, can be handled):",len(intersectionSet)

# pre-load the query term with its freq in the collection
queryTermWithFreqInCollectionDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freqInCollection = int(lineElements[1])
    if queryTerm not in queryTermWithFreqInCollectionDict:
        queryTermWithFreqInCollectionDict[queryTerm] = freqInCollection
    else:
        print "Unexpected Behaviour"
        exit(1)
print "len(queryTermWithFreqInCollectionDict):",len(queryTermWithFreqInCollectionDict)
inputFileHandler.close()

# step1: deal with the unseen query terms, need to give them one of the 5 ranges
# direct comparison with the length of the inverted list

# smallRangeID beginningLength,endingLength 
# [0,4]:       [1,100)            0.0000115262492427
# [5,9]:       [100,665)          0.000660982699686
# [10,25]:     [665,2473)         0.00474471376998
# [26,64]:     [2473,9964)        0.013327935727
# [65,999]:    [9964,25205179]    0.0274554562171
# [0,999]:     [1,25205179]       0.0000571680709718

queryTermAndProfProbablityDistributionDict = {}

counterForOutOfLexiconTerms = 0
counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 = 0
counterForTermsWhichHaveSeenFromThe10KQueries = 0
counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20 = 0
# For the new/unseen query terms
for queryTerm in newTermSet:
    if queryTerm not in queryTermWithRealFreqInTrainingQueriesDict or queryTermWithRealFreqInTrainingQueriesDict[queryTerm] < 20:
        
        # This component is for 2D
        if queryTermWithFreqInCollectionDict[queryTerm] >= 1 and queryTermWithFreqInCollectionDict[queryTerm] < 100:
            # final probability with the cellKey = "ROW1_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 2.7962236405996843e-10
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 100 and queryTermWithFreqInCollectionDict[queryTerm] < 665:
            # final probability with the cellKey = "ROW2_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 1.6040397629079132e-08
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 665 and queryTermWithFreqInCollectionDict[queryTerm] < 2473:
            # final probability with the cellKey = "ROW3_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 1.1538218688794478e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 2473 and queryTermWithFreqInCollectionDict[queryTerm] < 9964:
            # final probability with the cellKey = "ROW4_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 3.2578867212223025e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        elif queryTermWithFreqInCollectionDict[queryTerm] >= 9964 and queryTermWithFreqInCollectionDict[queryTerm] <= 25205179:
            # final probability with the cellKey = "ROW5_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 8.031888903939055e-07
            counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20 += 1
        else:
            # solution1
            # print "OutOfLexiconQueryTerm:",queryTerm
            
            # solution2
            # the rational behind this is: out of lexicon query terms has the smallest probability I can assign them.
            # the cellKey == "ROW1_0"
            queryTermAndProfProbablityDistributionDict[queryTerm] = 2.7962236405996843e-10
            counterForOutOfLexiconTerms += 1
        
        # This component is for transfer the probability from 2D to 1D
        # comment out the following sentence: 2D
        # NOT comment out the following sentence: 1D
        # queryTermAndProfProbablityDistributionDict[queryTerm] = 1.3870025585078873e-09
        
    else:
        # print "Popular query terms:",queryTerm,queryTermWithRealFreqInTrainingQueriesDict[queryTerm]
        queryTermAndProfProbablityDistributionDict[queryTerm] = queryTermWithRealFreqInTrainingQueriesDict[queryTerm] / NUM_QUERY_TERM_POSITIONS
        counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20 += 1

# For the query terms which have been seen from the training file
for queryTerm in intersectionSet:
    # directly copy the probability from queryTermWithAssociatedProbabilityDict to queryTermAndProfProbablityDistributionDict
    queryTermAndProfProbablityDistributionDict[queryTerm] = queryTermWithAssociatedProbabilityDict[queryTerm]
    counterForTermsWhichHaveSeenFromThe10KQueries += 1
       
print "counterForOutOfLexiconTerms:",counterForOutOfLexiconTerms
print "counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20:",counterForTermsWhichNOTAppearedIn85QueriesOrFreqLessThan20
print "counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20:",counterForTermsWhichAppearedIn85QueriesAndWithFreqGreaterOrEqual20
print "counterForTermsWhichHaveSeenFromThe10KQueries:",counterForTermsWhichHaveSeenFromThe10KQueries

print "len(queryTermAndProfProbablityDistributionDict):",len(queryTermAndProfProbablityDistributionDict)
print "len(queryTermsAndRealProbablityDistributionDict):",len(queryTermsAndRealProbablityDistributionDict)

'''
# Option1:
# The output component 
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheLatestProbabilitySettings1DProbabilityAdded"
outputFileHandler = open(outputFileName,"w")

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheLatestProbabilitySettings"
inputFileHandler = open(inputFileName,"r")
oldHeadLine = inputFileHandler.readline()
newHeadLine = oldHeadLine.strip() + " " + "ourProbabilityModel1D" + "\n"
outputFileHandler.write(newHeadLine)

# current output probability
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    newLine = line.strip() + " " + str( queryTermAndProfProbablityDistributionDict[queryTerm] ) + "\n"
    outputFileHandler.write(newLine)

inputFileHandler.close()
outputFileHandler.close()
'''

# Option2:
# The output component
# option1, 1D:
# outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom100KWithTheirTrueProbablityAndOurOwnModelPredictedProbablity1D"
# option2, 2D:
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom100KWithTheirTrueProbablityAndOurOwnModelPredictedProbablity2D"

outputFileHandler = open(outputFileName,"w")
for queryTerm in queryTermsAndRealProbablityDistributionDict:
    # + " " + str( queryTermsAndRealProbablityDistributionDict[queryTerm] )
    # + " " + str( queryTermAndProfProbablityDistributionDict[queryTerm] )
    outputFileHandler.write(queryTerm + " " + str( queryTermsAndRealProbablityDistributionDict[queryTerm] ) + " " + str( queryTermAndProfProbablityDistributionDict[queryTerm] ) + "\n")
outputFileHandler.close()

